package priv.lirenhe.dissertation.component.crawler_robot.crawler_actions.crawler_charset;

import java.io.UnsupportedEncodingException;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

import priv.lirenhe.dissertation.component.crawler_robot.crawler_actions.crawler_client.HttpClientCrawler;

public class CrawlerCharset {
	

	public static String changeCharset(String originWebContent) {
		String secondWebContent = null;
		String ChineseCharacter = null;
		
		secondWebContent = new String(originWebContent);
		String[] charsets ={"UTF-8","GBK","gb2312"};
		
		try {
			for(int i=0;i<charsets.length + 1;i++){
				ChineseCharacter = exeCode(secondWebContent);
				
				if(ChineseUtil.isMessyCode(ChineseCharacter) == true && i < charsets.length){
					secondWebContent = new String(originWebContent.getBytes("ISO-8859-1"),charsets[i]);//ISO-8859-1
				}else{
					originWebContent = secondWebContent;
					break;
				}
			}
		} catch (UnsupportedEncodingException e) {
			e.printStackTrace();
		}
		return originWebContent;
	}
	
	private static String exeCode(String webContent){
		String webContentText = null;
		String ChineseCharacter = null;
		
		Document doc = Jsoup.parse(webContent);
		webContentText = doc.text();
		if(webContentText.length() > 50 ){
			webContentText = webContentText.substring(0,50);
		}
		System.out.println("webContentText="+webContentText);
		ChineseCharacter = webContentText.replaceAll("[A-Z|a-z|(\\s\\d\\-\\(\\)\\[\\]/\\*)(\"\"'')(\\.,:?;<>{}+_\\&\\^%$#@!~`)(\\u00A0)]+", "");
		System.out.println("ChineseCharacter="+ChineseCharacter);
		return ChineseCharacter;
	}

	
}
